import requests
from bs4 import BeautifulSoup
import time
import random
from fake_useragent import UserAgent  # 随机UA
import pandas as pd  # 数据存储
import re  # 添加 re 模块导入
import json  # 添加 json 模块导入

class WebScraper:

    def __init__(self):
        self.base_url = "http://xzqh.mca.gov.cn"
        self.headers = {
          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'Upgrade-Insecure-Requests': '1',
            'Sec-Fetch-Dest': 'document',
            'Sec-Fetch-Mode': 'navigate',
            'Sec-Fetch-Site': 'none',
            'Sec-Fetch-User': '?1',
            'Cache-Control': 'max-age=0',
        }
        self.session = requests.Session()

    def get_address_info(self):
        try:
            # 1. 获取并解析页面中的省级数据
            response = self.session.get(f"{self.base_url}/map",headers=self.headers,timeout=10,verify=True)
            response.encoding = 'gbk'  # 设置正确的编码
            
            # form表单格式为：<form name="fsform"><table class="select_table"><tr><td><input type="hidden" id="pyArr" value="具体内容"></td></tr></table></form>
            soup = BeautifulSoup(response.text, 'html.parser')
            form = soup.find('form', {'name': 'fsform'})
            if form:
                input_data = form.find('input', {'id': 'pyArr'})
                if input_data:
                    json_data =  input_data.get('value')
                else:
                    print("未找到input标签")
            else:
                print("未找到form表单")

            if json_data:
                info_array = json.loads(json_data)
            else:
                print("未找到json数据")
            
            return info_array
        
        except Exception as e:
            print(f"爬取失败: {e}")
            raise
    
    def handle_address_info(self,info_array):

        province_list = []
        city_list = []
        new_city_list = []
        area_list = []
        for info in info_array:
            if info['code'].endswith('0000'):
                province_list.append(info)
            elif info['code'].endswith('00'):
                info['parantCode'] = info['code'][:2] + '0000'
                city_list.append(info)
            else:
                info['parantCode'] = info['code'][:4] + '00'
                area_list.append(info)

        for province in province_list:
            province_city_list = [city for city in city_list if city['parantCode'] == province['code']]
            print(f"通过省份查找城市: {province_city_list}")
            if province_city_list == []:
                # 如果直接用list(province)会造成数据结构错误，只获取了key，
                # "cityList": [
                # "cName",
                # "code",
                # "py",
                # "jp",
                # "qp"
                # ]
                province_city_data = {
                    'cName': province['cName'],
                    'code': province['code'][:2] + '0100',  # 生成市级代码
                    'py': province['py'],
                    'jp': province['jp'],
                    'qp': province['qp']
                }
                province['cityList'] = [province_city_data]
                new_city_list.append(province_city_data)
            else:
                province['cityList'] = province_city_list

        city_list.extend(new_city_list)

        for city in city_list:
            city_area_list = [area for area in area_list if area['parantCode'] == city['code']]
            print(f"通过城市查找区县: {city_area_list}")
            if city_area_list == []:
                city['areaList'] = [{
                    'cName': city['cName'],
                    'code': city['code'][:4] + '01',  # 生成区县代码
                    'py': city['py'],
                    'jp': city['jp'],
                    'qp': city['qp']
                }]
            else:
                city['areaList'] = city_area_list
        return province_list


    def save_to_json(self, data, filename='output.json'):

        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=4)    

    def save_to_json_list(self,data_list, file_path='output_list.json'):

        try:
            # 确保转换成list类型
            if hasattr(data_list, '__iter__') and not isinstance(data_list, (dict, str)):
                data_list = list(data_list)
            
            with open(file_path, 'w', encoding='utf-8') as f:
                # 使用ensure_ascii=False来正确处理中文
                # indent=2 使输出格式更易读
                json.dump(data_list, f, ensure_ascii=False, indent=2)
        except Exception as e:
            print(f"保存数据失败: {e}")
            raise

    def save_to_csv(self, data, filename='output.csv'):
        """
        保存数据到CSV文件
        Args:
            data: 要保存的数据列表
            filename: 保存的文件名
        Raises:
            ValueError: 当数据为空时抛出
        """
        try:
            print(f"正在保存数据到CSV文件: {filename}")
            
            # 检查数据是否为空
            if not data:
                raise ValueError(f"数据为空，无法保存到 {filename}")
                
            # 检查数据类型
            if not isinstance(data, (list, tuple)):
                raise TypeError(f"数据类型错误，期望list或tuple，实际是 {type(data)}")
                
            # 打印数据统计信息
            print(f"数据条数: {len(data)}")
            
            # 转换为DataFrame并保存
            df = pd.DataFrame(data)
            
            # 检查DataFrame是否为空
            if df.empty:
                raise ValueError(f"转换后的DataFrame为空，无法保存到 {filename}")
                
            # 保存数据
            df.to_csv(filename, index=False, encoding='utf-8-sig')
            print(f"数据已成功保存到 {filename}")
            
        except Exception as e:
            print(f"保存数据到 {filename} 失败: {e}")
            print(f"数据内容: {data}")  # 打印数据内容以便调试
            raise  # 重新抛出异常


